# Probability of discordant pair.
calc.ranking.cost.1 <- function(data, y.hat){
  data$y.hat <- y.hat;
  b <- by(data, data$qid, 
          function(x){
            count <- 0;
            for(i in 1:nrow(x)){
              votes <- x$votes[i];
              y.hat <- x$y.hat[i];
              count <- count + sum(x$votes>votes & x$y.hat<y.hat);
            }
            return(count);
          })
  b2 <- by(data, data$qid, 
          function(x){
            count <- 0;
            for(i in 1:nrow(x)){
              votes <- x$votes[i];
              count <- count + sum(x$votes>votes);
            }
            return(count);
          })
  return(1-sum(b)/sum(b2));
}

# probability that actual best is among top k predicted best
calc.ranking.cost.2 <- function(data, y.hat, k){
  data$y.hat <- y.hat;
  b <- by(data, data$qid, 
          function(x){
            i <- which.max(x$votes);
            return(sum(x$y.hat>=x$y.hat[i])<=k);
          })
  return(mean(b));
}

# Average predicted rank of top actual best answer.
calc.ranking.cost.3 <- function(data, y.hat){
  data$y.hat <- y.hat;
  b <- by(data, data$qid, 
          function(x){
            i <- which.max(x$votes);
            return(sum(x$y.hat>=x$y.hat[i]));
          })
  return(mean(b));
}

# Average predicted rank of top actual best answer on scale 0-1
calc.ranking.cost.4 <- function(data, y.hat){
  data$y.hat <- y.hat;
  b <- by(data, data$qid, 
          function(x){
            i <- which.max(x$votes);
            return(
              (sum(x$y.hat>=x$y.hat[i])-1)
              /(nrow(x)-1)
               );
          })
  return(mean(b));
}


calc.ranking.cost.5 <- function(data, y.hat){
  data$y.hat <- y.hat;
  b <- by(data, data$qid, 
          function(x){
            count <- 0;
            for(i in 1:nrow(x)){
              votes <- x$votes[i];
              y.hat <- x$y.hat[i];
              count <- count + sum(x$votes>votes & x$y.hat<y.hat);
            }
            return(count);
          })
  b2 <- by(data, data$qid, 
           function(x){
             count <- 0;
             for(i in 1:nrow(x)){
               votes <- x$votes[i];
               count <- count + sum(x$votes>votes);
             }
             return(count);
           })
  return(b);
}
